import copy
import re
import threading
import time

import requests

from dataInsertDB import cxOracleDB
from log.clog import *
from utils.sheettools import dataAppend
from utils.tools import MUSTEXDICT, EXREGION, EXDICT, COREGION, CONCLUEDEDICT, DISTURBDICT, \
    DEVELOP_WINNING_BID_DATA, DEVELOP_BIDDING_DATA, ENSUREFLAG, NULLFLAG, DICT_WIN_BID_DATA, \
    DICT_BIDDING_DATA, NOT_DICT_WIN_BID_DATA, NOT_DICT_BIDDING_DATA, CONSTDATA
import asyncio
from PageAnalyse.singlePageAnalyse import singlePageAnalyse


class ztbcrawler(cxOracleDB, singlePageAnalyse):
    # 初始化定义
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.mainUrl = 'http://ztb.guizhou.gov.cn/trade/?category=affiche'
        self.REGION = [5200, 5201, 520900, 520101, 520001, 520111]
        self.TENDERTYPE = ['affiche', 'publicity']
        self.lock = threading.Lock()

    # ztb 网站 header 定义
    def ztbHeaderDef(self, tendertype, region, starttime, endtime, pageindex=1):
        cookies = {
            'acw_tc': '784c10e817228545085101049e538b4b119343a17b6845bd943745bf6e9b37',
            '_d_id': '70ef9d84193c82273909e26a782e7e',
            'Hm_lvt_5f50c10002e72d2f5aa3e0563594f600': '1721179572,1721265521,1722216732,1722854509',
            'HMACCOUNT': '90352FC2C966B0C8',
            'acw_sc__v3': '66b0aeffda4c0e00804fe4fa52179bdee615f38c',
            'tfstk': 'fnXrTzfObpbfqDCHuE9e0k7lbe9JpdU_89TBxMxykUmBRWCq8G_rAw9WVsuFoGOhFM9kxBbCS_TlLMHexMbL2eZ8GgIJpp4_56y12gdfqhblU3jmmiKpxb4lpcFwpp4_ljMHwRJd5FAghJS003tsKpAHZoJDYEDkxYxnoo-poevHrYDcmh-nE0xHtU-F-aH2uQqNNd6jgvLkaFj26vDeJAOJSCTxKnB24eYGqEknKUezXbSh-ykBI1QVLCbLLxp5bsvD4srniUANZatA8RkkrtSlEU6ulYTleif1dZqnKe5csC8lMrepZ1Q5sIWbzxLGiiSBgsEKNe19xwdOdPHJrMjOB19aLD-V_ivV4AupmWXApMmrTQxv0FZ4m_QP4hZCE7ai9Xd4Mn8_XlhK9QY60FZq3XhpgZK25urh.',
            'Hm_lpvt_5f50c10002e72d2f5aa3e0563594f600': '1722856237',
        }
        headers = {
            'Accept': '*/*',
            'Referer': self.mainUrl,
            'X-Requested-With': 'XMLHttpRequest',
            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1',
        }

        params = {
            'pubDate': starttime + '|' + endtime,
            'pubType': 'all',
            'region': region,
            'industry': 'all',
            'prjType': 'all',
            'noticeType': tendertype,
            'noticeClassify': 'all',
            'pageIndex': pageindex,
            'args': '',
        }

        try:
            response = requests.get('http://ztb.guizhou.gov.cn/api/trade/search',
                                    cookies=cookies,
                                    params=params, headers=headers,
                                    verify=False)
        except Exception as e:
            # print(e)
            return '', False
        print(response.json()['totalPage'], response.json()['data'])
        return response.json()['totalPage'], response.json()['data']

    # 对 ztb 网站的单个页面发起请求的 Header
    async def ztbSingleCrawler(self, id):
        url = 'http://ztb.guizhou.gov.cn/trade/bulletin/?id=' + str(id)
        cookies = {
            'Hm_lvt_5f50c10002e72d2f5aa3e0563594f600': '1707063135,1707444797,1708446777',
            '_d_id': '5cea1f2cac41940878f9246f93e652',
            'Hm_lpvt_5f50c10002e72d2f5aa3e0563594f600': '1708879637',
        }

        headers = {
            'Accept': '*/*',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            # 'Cookie': 'Hm_lvt_5f50c10002e72d2f5aa3e0563594f600=1707063135,1707444797,1708446777; _d_id=5cea1f2cac41940878f9246f93e652; Hm_lpvt_5f50c10002e72d2f5aa3e0563594f600=1708879637',
            'Proxy-Connection': 'keep-alive',
            'Referer': f'http://ztb.guizhou.gov.cn/trade/bulletin/?id={id}',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            'X-Requested-With': 'XMLHttpRequest',
        }
        try:
            res = requests.get(f'http://ztb.guizhou.gov.cn/api/trade/{str(id)}',
                               cookies=cookies,
                               headers=headers,
                               verify=False)
            res = res.json()

        except Exception as e:
            logger.exception('解析错误', e, url)
        finally:
            return res, url

    def isInDict(self, patt, text):
        try:
            # 使用具体的异常类型代替裸的except
            match = re.search(patt, text, flags=re.MULTILINE | re.DOTALL)
            if match:
                return True
            else:
                return False
        except re.error as e:
            print(f"正则表达式错误: {e}")
            raise e

    # 正则表达式统一入口
    def matchingJudgment(self, patt, text):
        try:
            for pat in patt.values():
                res = re.search(pat, text, flags=re.MULTILINE | re.DOTALL)
                if res:
                    resbody = res.group(1)
                    return resbody
        except:
            resbody = ' '
            return resbody

    # 招标金额匹配
    def matchBidAmount(self, text):
        patt = {'patt1': r'采购预算.*?\">(\d+.*?)<',
                'patt2': r'预算金额.*?([0-9,.]+)元',
                'patt3': r'采购预算.*?([0-9,.]+)元',
                'patt4': r'项目预算.*?([0-9,.]+)元',
                'patt5': r'最高限价.*?([0-9,.]+)元'}
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 招标人匹配
    def senderMatch(self, text):
        data = '公司|局|厅|监测院|学院|分行|图书馆|管理层'
        patt = {
            'patt5': rf'采购人名称：.*?([\u4e00-\u9fa5]+({data}))',
            'patt5': rf'采购人名称:.*?([\u4e00-\u9fa5]+({data}))',
            'patt6': rf'采购人信息：.*?([\u4e00-\u9fa5]+({data}))',
            'patt7': rf'采购人.*?([\u4e00-\u9fa5]+({data}))',
            'patt8': rf'招标人.*?([\u4e00-\u9fa5]+({data}))',
        }
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 中标人匹配
    def theWinningBidderIsMatched(self, text):
        patt = {
            'patt1': r'(?<=[A-Z0-9]{19})(.*?)(?=总价人民币|成交供应商)',
            'patt2': r'第一中标候选人：.*?([\u4e00-\u9fa5]+(公司|局|厅|监测院|学院|分行|图书馆))'
        }
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 中标金额匹配
    def tenderPrice(self, text):
        patt = {
            'patt1': r'价钱.*?>(.*?)<',
            'patt2': r'投标报价.*?([0-9,.]+).?元',
            'patt3': r'报价.*?([0-9,.]+).?元'
                }
        res = self.matchingJudgment(patt=patt, text=text)
        return res

    # 单页面分析
    def singlePageAnalyse(self, winType, text):
        text = super().getContent(text)
        sender = self.senderMatch(text)
        if winType:
            winBidder = self.theWinningBidderIsMatched(text)
            money = self.tenderPrice(text)
            if money != '-':
                money = super().convert_to_ten_thousand_unit(money)
            return money, sender, winBidder
        else:
            money = self.matchBidAmount(text)
        if money != '-':
            money = super().convert_to_ten_thousand_unit(money)
        return money, sender

    # 时间格式化
    def timeConvert(self, times):
        pass

    # ztb 网站的主要分析逻辑代码
    def ztbMainAnalyse(self, tendertype, region):
        """
        招标类型
            affiche: 招标公告
            publicity:中标公告
        区域:
            5200 贵州省
            5201 贵阳市
        'pubDate': '2024-02-20|2024-02-23',
        :return:
        """
        SEARCHDICT = {
            5200: '贵州省',
            5201: '贵阳市',
            520900: '贵安新区',
            'affiche': '招标公告',
            'publicity': '中标公告'
        }
        DESIGN_DICT_FLAG = False
        # print(tendertype)
        if tendertype == 'publicity':
            WinTheBidOrNot = True
        else:
            WinTheBidOrNot = False
        time.sleep(self.config.sleep_time)

        # print(region)
        time.sleep(self.config.sleep_time)
        totalpage, flag = self.ztbHeaderDef(tendertype=tendertype, region=region,
                                            starttime=self.config.start_time,
                                            endtime=self.config.end_time)
        # 循环处理总的页数
        for pageindex in range(1, totalpage):
            time.sleep(self.config.sleep_time)
            flag, menuPageData = self.ztbHeaderDef(tendertype=tendertype, region=region,
                                                   starttime=self.config.start_time,
                                                   endtime=self.config.end_time, pageindex=pageindex)
            if not flag:
                continue
            # print( f'当前爬取区域为:{SEARCHDICT[region]},代码为:{region} 当前爬取类型为:{SEARCHDICT[tendertype]},当前爬取页数为:{pageindex}')
            # 循环处理每一个目录页的数据
            for singleData in menuPageData:
                DICTTYPE = False
                time.sleep(self.config.sleep_time)
                outOfLoopCondition = False
                # 判断是否重复
                for saveTitleInClass in self.config.titleContainer:
                    if saveTitleInClass == singleData['Title']:
                        outOfLoopCondition = True
                if outOfLoopCondition:
                    continue
                # 判断是否为贵州省/贵阳市项目
                coregion = COREGION
                for conregion in coregion:
                    # print(conregion,singleData['RegionName'])
                    if conregion != singleData['RegionName']:
                        outOfLoopCondition = True
                        continue
                    if conregion == singleData['RegionName']:
                        outOfLoopCondition = False
                        break
                if outOfLoopCondition:
                    print('区域不符合', ' code:', conregion, '| bid:', singleData['RegionName'],
                          singleData['Title'])
                    continue
                # print(singleData['Title'])
                # 判断是否在必须排除列表中
                for singleMustExDict in MUSTEXDICT:
                    if singleMustExDict in singleData['Title']:
                        outOfLoopCondition = True
                        break
                    if singleMustExDict in singleData['BTypeName']:
                        outOfLoopCondition = True
                        break
                if outOfLoopCondition:
                    print('必须排除列表中', 'code:', singleMustExDict, '| bid:', singleData['Title'], '|',
                          singleData['BTypeName'])
                    continue
                # 判断是否在排除字典中
                for singleEx in EXDICT:
                    if singleEx in singleData['Title']:
                        outOfLoopCondition = True
                        break
                # 删除干扰关键字
                saveTitle = singleData['Title']
                for disturbkeyword in DISTURBDICT:
                    if disturbkeyword in singleData['Title']:
                        singleData['Title'] = singleData['Title'].replace(disturbkeyword, '')
                datadict = {}
                try:
                    pattern = '|'.join(map(re.escape, CONCLUEDEDICT))
                    # 使用正则表达式查找匹配
                    matches = re.findall(pattern, singleData['Title'])
                    # 如果有匹配项，执行语句块
                    if matches:
                        for match in matches:
                            sadata = match
                        outOfLoopCondition = False
                        DICTTYPE = True
                        datadict['DICT'] = ENSUREFLAG
                        datadict['keyword'] = sadata
                    else:
                        datadict['DICT'] = NULLFLAG
                        datadict['keyword'] = NULLFLAG
                except Exception as e:
                    raise e

                if outOfLoopCondition:
                    print('排除字典中', singleEx, singleData['Title'])
                    continue
                self.config.titleContainer.append(singleData['Title'])
                # 抓取单页面数据并进行数据处理分析
                singlePageData, url = asyncio.run(self.ztbSingleCrawler(singleData['Id']))
                if not url:
                    continue
                # title'标题名称' 处理及异常判断
                try:
                    datadict['title'] = saveTitle
                except Exception as e:
                    datadict['title'] = ' '
                    # print(e, 'title error')
                    raise e
                # districtName'地区名称' 处理及异常判断
                try:
                    datadict['districtName'] = singleData['RegionName']
                except Exception as e:
                    datadict['districtName'] = ' '
                    # print(e, 'districtName error')
                    raise e
                # publishDate'发布时间' 处理及异常判断
                try:
                    datadict['publishDate'] = singlePageData['PublishDate']
                except Exception as e:
                    datadict['publishDate'] = ' '
                    print(e, 'publishDate error')

                datadict['url'] = url
                if WinTheBidOrNot:
                    money, sender, winBidder = self.singlePageAnalyse(winType=WinTheBidOrNot,
                                                                      text=singlePageData['Content'])
                    datadict['money'] = money
                    datadict['sender'] = sender
                    datadict['winBidder'] = winBidder
                else:
                    money, sender = self.singlePageAnalyse(winType=WinTheBidOrNot,
                                                           text=singlePageData['Content'])
                    datadict['money'] = money
                    datadict['sender'] = sender

                    '''!important
                        :description: 判断是否为设计咨询标类型数据,并保存设计咨询标数据
                    '''
                    for key in CONSTDATA.DESIGN_DICT:
                        if self.isInDict(patt=key,
                                         text=datadict['title']):
                            DESIGN_DICT_FLAG = True
                            if DESIGN_DICT_FLAG:
                                break
                        else:
                            DESIGN_DICT_FLAG = False

                if WinTheBidOrNot:
                    if DICTTYPE and DESIGN_DICT_FLAG:
                        sheetname = CONSTDATA.DESIGN_WIN_BID_DATA
                    elif DICTTYPE:
                        sheetname = DICT_WIN_BID_DATA
                    else:
                        sheetname = NOT_DICT_WIN_BID_DATA
                    datatemp = [datadict['DICT'], datadict['keyword'], datadict['districtName'],
                                datadict['title'],
                                datadict['sender'],
                                datadict['winBidder'], datadict['publishDate'], datadict['money'],
                                datadict['url']]
                    self.lock.acquire()
                    alldatatemp = copy.deepcopy(datatemp)
                    dataAppend(data=datatemp, worrbook=self.config.file_path, sheetname=sheetname)

                    alldatatemp.append(singlePageData['Content'])
                    dataAppend(data=alldatatemp, worrbook=self.config.file_path, sheetname=DEVELOP_WINNING_BID_DATA)
                    self.lock.release()

                else:
                    if DICTTYPE and DESIGN_DICT_FLAG:
                        sheetname = CONSTDATA.DESIGN_BIDDING_DATA
                    elif DICTTYPE:
                        sheetname = DICT_BIDDING_DATA
                    else:
                        sheetname = NOT_DICT_BIDDING_DATA
                    datatemp = [datadict['DICT'], datadict['keyword'], datadict['districtName'],
                                datadict['title'],
                                datadict['sender'],
                                datadict['publishDate'], datadict['money'], datadict['url']]

                    self.lock.acquire()
                    alldatatemp = copy.deepcopy(datatemp)
                    dataAppend(data=datatemp, worrbook=self.config.file_path, sheetname=sheetname)

                    alldatatemp.append(singlePageData['Content'])
                    dataAppend(data=alldatatemp, worrbook=self.config.file_path, sheetname=DEVELOP_BIDDING_DATA)
                    self.lock.release()
                    # super().queryData()

    # 线程入口
    async def threadMain(self):
        import logging

        # 创建或获取一个 logger
        logger = logging.getLogger(__name__)

        # 设置日志级别
        logger.setLevel(logging.ERROR)

        # 创建一个文件处理器，将日志记录到 app.log 文件中
        file_handler = logging.FileHandler('app.log')
        file_handler.setLevel(logging.ERROR)

        # 创建一个格式化器
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        file_handler.setFormatter(formatter)

        # 将处理器添加到 logger 中
        logger.addHandler(file_handler)

        threadList = []
        for i in self.TENDERTYPE:
            for j in self.REGION:
                try:
                    thread = threading.Thread(target=self.ztbMainAnalyse, args=(i, j))
                    threadList.append(thread)
                    thread.start()
                except Exception as e:
                    logger.exception(e)

        for threadsingle in threadList:
            threadsingle.join()
